﻿using SPWebShell;
using SPWebShell.entity;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web.Services.Protocols;
using System.Xml;

namespace SPWebShell
{
    public class SPRepositoryImplementation : ISPRepository
    {
        int _batchsize;
        Action<string> _verboselogger;
        Action<string> _errorLogger;
        Stopwatch _sw;
        public SPRepositoryImplementation(Action<string> verboselogger, Action<string> errorlogger)
        {
            _batchsize = 100;
            _verboselogger = verboselogger;
            _errorLogger = errorlogger;
        }
        public SPRepositoryImplementation()
        {
            _batchsize = 100;
        }
        public string GetContainerWebFromDocLibUrl(string fulldocliburl)
        {
            LogVerboseMessage("Resolving container web site from full URL:{0}", fulldocliburl);
            SPWebShell.SPWebs.Webs web = new SPWebShell.SPWebs.Webs();
            //web.Url=
            Util util = new Util();
            string serveronlyurl=util.GetServerWithSchemeAndPort(fulldocliburl);
            if (!serveronlyurl.EndsWith("/")) serveronlyurl = serveronlyurl + "/";
            web.Url = serveronlyurl + "_vti_bin/webs.asmx";
            web.Credentials = System.Net.CredentialCache.DefaultCredentials;
            string urlWeb=web.WebUrlFromPageUrl(fulldocliburl);
            LogVerboseMessage("URL to web site:{0}",urlWeb);
            return urlWeb;
        }
        public SPDoclibLocation ResolveDoclibUrlFromPageUrl(string urlAbsoluteToAnyItemInDoclib)
        {
            LogVerboseMessage("Resolving document library from URL:{0}", urlAbsoluteToAnyItemInDoclib);
            SPDoclibLocation loc = new SPDoclibLocation();
            string urlContainerWeb = this.GetContainerWebFromDocLibUrl(urlAbsoluteToAnyItemInDoclib);
            loc.SiteUrl = urlContainerWeb;
            Uri uAbs = new Uri(urlAbsoluteToAnyItemInDoclib,true);
            Uri uContainer = new Uri(urlContainerWeb, true);
            string t1 = uAbs.PathAndQuery.Substring(uContainer.PathAndQuery.Length);
            if (t1.StartsWith("/")) t1 = t1.Substring(1);
            int slashpos = t1.IndexOf("/");
            string doclibname="";
            if (slashpos >= 0)
                doclibname = t1.Substring(0, slashpos);
            else
                doclibname = t1;
            loc.DoclibName = doclibname;
            LogVerboseMessage("doclib name:{0}",loc.DoclibName);
            return loc;
        }
        public SPDoclibItem[] GetAllItems(SPDoclibLocation location, int startid)
        {
            LogVerboseMessage("GetAllItems,{0},{1}", location.SiteUrl, location.DoclibName);
            string weburl = location.SiteUrl;
            string doclib=location.DoclibName;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlDocument xmlDoc = new System.Xml.XmlDocument();

            XmlNode ndQuery = xmlDoc.CreateNode(XmlNodeType.Element, "Query", "");
            XmlNode ndViewFields = xmlDoc.CreateNode(XmlNodeType.Element, "ViewFields", "");
            XmlNode ndQueryOptions =
                xmlDoc.CreateNode(XmlNodeType.Element, "QueryOptions", "");
            //<Folder></Folder>
            ndQueryOptions.InnerXml =
                "<ViewAttributes Scope='RecursiveAll'/>"+
                "<IncludeMandatoryColumns>FALSE</IncludeMandatoryColumns>" +
                "<DateInUtc>TRUE</DateInUtc>";
            ndViewFields.InnerXml = (new string[] { 
                                Constants.FIELD_CREATED, Constants.FIELD_MODIFIED , Constants.FIELD_VERSION , 
                                Constants.FIELD_VERSIONSTRING , Constants.FIELD_ITEMTYPE,Constants.FIELD_FILESIZE,
                                Constants.FIELD_TITLE,Constants.FIELD_ENCODEDABSURL}).ConvertToViewFields();
            ndQuery.InnerXml = string.Format("<Where><Gt><FieldRef Name='ID'/><Value Type='Counter'>{0}</Value></Gt>" + "</Where>",startid);

            string batchsize = this.BatchSize.ToString();
            XmlNode ndListItems = lists.GetListItems(doclib, null, ndQuery, ndViewFields, batchsize, ndQueryOptions, "");
            XmlNamespaceManager ns = new XmlNamespaceManager(ndListItems.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNodeList ndRows = ndListItems.SelectNodes(".//z:row", ns);
            List<SPDoclibItem> items = new List<SPDoclibItem>();
            string docliburlpath = new Uri(location.SiteUrl).PathAndQuery;
            foreach (XmlNode ndRow in ndRows)
            {
                string outerxml = ndRow.OuterXml;
                SPDoclibItem item = null;
                try
                {
                    item = ndRow.NodeToDoclibItem();
                    item.RelativeUrl = item.ServerRelativeUrl.Substring(docliburlpath.Length);                    
                    items.Add(item);
                }
                catch (Exception ex)
                {
                    LogError("Error while parsing SP List Item:");
                    LogError(outerxml);
                    LogError(ex.ToString());
                }
            }
            return items.ToArray();
        }
        /// <summary>
        /// Used for updating the Title field of the uploaded item
        /// </summary>
        /// <param name="location"></param>
        /// <param name="id"></param>
        /// <param name="title"></param>
        private void UpdateTitle(SPDoclibLocation doclibLocation, int spid, string title)
        {
            LogVerboseMessage("Updating the Title of the uploaded document to '{0}'", title);
            string weburl = doclibLocation.SiteUrl;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlNode ndResults = null;
            string xmlbatch = string.Format(
                "<Batch><Method ID='1' Cmd='Update' ><Field Name='ID'>{0}</Field><Field Name='Title'>{1}</Field></Method></Batch>",
                spid, title);
            XmlDocument xmlUpdateTitle = new XmlDocument();
            xmlUpdateTitle.LoadXml(xmlbatch);
            ndResults = lists.UpdateListItems(doclibLocation.DoclibName, xmlUpdateTitle);                
        }
        public SPDoclibItem CreateFolder(SPDoclibLocation doclibLocation, string folderName, string containingfolder)
        {
            LogVerboseMessage("CreateFolder,{0}", folderName);
            string weburl=doclibLocation.SiteUrl;           
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlNode ndResults = null;

            if (string.IsNullOrWhiteSpace(containingfolder))
            {
                //create the folder at the top level
                string xmlbatch = string.Format(
                    "<Batch><Method ID='1' Cmd='New' ><Field Name='FSObjType'>1</Field><Field Name='BaseName'>{0}</Field></Method></Batch>",
                    folderName);
                XmlDocument docCreateItem = new XmlDocument();
                docCreateItem.LoadXml(xmlbatch);
                ndResults=lists.UpdateListItems(doclibLocation.DoclibName, docCreateItem);                
            }
            else
            {
                containingfolder=containingfolder.Trim();
                if (!containingfolder.StartsWith("/")) containingfolder = "/" + containingfolder;
                if (containingfolder.EndsWith("/")) containingfolder = containingfolder.Substring(0, containingfolder.Length - 1);

                string rootFolder = string.Format("{0}{1}{2}", weburl,doclibLocation.DoclibName,containingfolder);
                string xmlbatch = string.Format(
                    "<Batch RootFolder='{1}'><Method ID='1' Cmd='New' ><Field Name='FSObjType'>1</Field><Field Name='BaseName'>{0}</Field></Method></Batch>",
                    folderName, rootFolder); //urlContainingFolders , TODO test why error message is not extractable when this parameter is used
                XmlDocument docCreateItem = new XmlDocument();
                docCreateItem.LoadXml(xmlbatch);
                ndResults = lists.UpdateListItems(doclibLocation.DoclibName, docCreateItem);
            }
            string message="";
            bool iserror = ndResults.IsSPError(out message);
            if (iserror) throw new Exception(string.Format("Unable to create folder:{0}\r\n{1}", folderName, message));
            SPDoclibItem newItem = new SPDoclibItem();
            XmlNamespaceManager ns = new XmlNamespaceManager(ndResults.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNodeList ndUserRows = ndResults.SelectNodes(".//z:row ", ns);
            if (ndUserRows.Count == 0) throw new Exception("Error while creating folder. No records were updated");
            XmlNode ndRow0 = ndUserRows[0];
            //newItem.IsFolder = true;
            //newItem.SPID = ndRow0.SafeGetXmlAttribute("ID").ToInt();
            //newItem.Created = ndRow0.SafeGetXmlAttribute(Constants.FIELD_CREATED).ToDate();
            //newItem.CreatedBy = "";
            //newItem.Modified = ndRow0.SafeGetXmlAttribute(Constants.FIELD_CREATED).ToDate();
            //newItem.ModifiedBy = "";
            //newItem.FileSize = 0;
            //newItem.Name = ndRow0.SafeGetXmlAttributeRemoveID("FileLeafRef");
            //newItem.ServerRelativeUrl = ndRow0.SafeGetXmlAttributeRemoveID("FileRef");
            //newItem.Title = ndRow0.SafeGetXmlAttribute("Title");
            //string encodedAbsUrl = ndRow0.SafeGetXmlAttribute("EncodedAbsUrl");
            //newItem.RelativeUrl = encodedAbsUrl.Substring(weburl.Length);
            newItem = ndRow0.NodeToDoclibItem();
            //string docliburlpath = new Uri(doclibLocation.SiteUrl).PathAndQuery;
            //newItem.RelativeUrl = newItem.ServerRelativeUrl.Substring(docliburlpath.Length);
            //SPListItem[] itemsRequery=this.GetListItems(doclibLocation, newItem.SPID, 1, null, false, "");
            //SPListItem thisItem=itemsRequery.First(i=>(i.SPID==newItem.SPID));
            //SPDoclibItem newfolder = new SPDoclibItem();
            //newfolder.
            return newItem;//TODO the list item when updated gets back Create/Modified in local regional setting.
            //YOu need to do a requery and get it back in UTC.
        }
        public void UploadFile(byte[] filebytes, SPDoclibLocation doclibDestination, string fileName, string folderpath,string title)
        {
            LogVerboseMessage("UploadFile,{0},{1},{2}", doclibDestination.SiteUrl, folderpath, fileName);
            try
            {
                string urlFullNewItem = doclibDestination.SiteUrl.Trim();
                if (!urlFullNewItem.EndsWith("/")) urlFullNewItem = urlFullNewItem + "/";
                if (string.IsNullOrEmpty(fileName)) throw new ArgumentNullException("The name of the file cannot be blank");
                string urlRelative = "";
                if (string.IsNullOrWhiteSpace(folderpath))
                {
                    urlFullNewItem = urlFullNewItem + doclibDestination.DoclibName + "/" + fileName;
                }
                else
                {
                    urlRelative = folderpath;
                    if (urlRelative.StartsWith("/")) urlRelative = urlRelative.Substring(1);//eliminate leading slash
                    urlFullNewItem = urlFullNewItem + doclibDestination.DoclibName + "/" + urlRelative + "/" + fileName;
                }
                WebRequest request = WebRequest.Create(urlFullNewItem);
                request.Credentials = CredentialCache.DefaultCredentials;
                request.Method = "PUT";
                using (Stream stream = request.GetRequestStream())
                {
                    stream.Write(filebytes, 0, filebytes.Length);
                }
                WebResponse response = null;
                response = request.GetResponse();
                ///
                /// Update the title field of the document
                ///
                if (!string.IsNullOrWhiteSpace(title))
                {
                    string key = "ETag";
                    try
                    {
                        string etag = response.Headers.Get(key);
                        Regex r = new Regex(@"\{.{1,}\}");//get the GUID
                        Match m = r.Match(etag);
                        string uniqueid = "";
                        if (m.Success)
                        {
                            uniqueid = m.ToString();
                        }
                        else
                        {
                            LogError("Coul not parse Etag after uploading document:{0}", urlFullNewItem);
                        }
                        LogVerboseMessage("UniqueID of uploaded document:{0}", uniqueid);
                        Guid id = new Guid(uniqueid);
                        SPDoclibItem newitem = this.GetItemUsingUniqueId(doclibDestination, id);//this works
                        UpdateTitle(doclibDestination, newitem.SPID, title);
                    }
                    catch (Exception ex)
                    {
                        LogError("Etag header not found in Response after uploading document:{0}", urlFullNewItem);
                        LogError("Title field has not got updated for this item");
                        LogError(ex.ToString());
                    }
                }
            }
            catch (WebException wex)
            {
                if (wex.Response != null) 
                {
                    using (var errorResponse = (HttpWebResponse)wex.Response)
                    {
                        using (var reader = new StreamReader(errorResponse.GetResponseStream()))
                        {
                            HttpStatusCode code = ((System.Net.HttpWebResponse)(wex.Response)).StatusCode;
                            string error = reader.ReadToEnd();
                            throw new Exception(string.Format("Error while uploading file:{0}\\{1}\r\n{2}\r\n{3}",folderpath,fileName,code,error));
                        }
                    }
                }
            }
        }
        private SPDoclibItem GetItemUsingUniqueId(SPDoclibLocation location,Guid id)
        {
            string weburl = location.SiteUrl;
            string doclib = location.DoclibName;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlDocument xmlDoc = new System.Xml.XmlDocument();

            XmlNode ndQuery = xmlDoc.CreateNode(XmlNodeType.Element, "Query", "");
            XmlNode ndViewFields = xmlDoc.CreateNode(XmlNodeType.Element, "ViewFields", "");
            XmlNode ndQueryOptions =
                xmlDoc.CreateNode(XmlNodeType.Element, "QueryOptions", "");
            //<Folder></Folder>
            ndQueryOptions.InnerXml =
                "<ViewAttributes Scope='RecursiveAll'/>" +
                "<IncludeMandatoryColumns>FALSE</IncludeMandatoryColumns>" +
                "<DateInUtc>TRUE</DateInUtc>";
            ndViewFields.InnerXml = (new string[] { 
                                Constants.FIELD_CREATED, Constants.FIELD_MODIFIED , Constants.FIELD_VERSION , 
                                Constants.FIELD_VERSIONSTRING , Constants.FIELD_ITEMTYPE,Constants.FIELD_FILESIZE,
                                Constants.FIELD_TITLE,Constants.FIELD_ENCODEDABSURL}).ConvertToViewFields();
            ndQuery.InnerXml = string.Format("<Where><Eq><FieldRef Name='UniqueId'/><Value Type='Lookup'>{0}</Value></Eq>" + "</Where>", id.ToString());

            string batchsize = this.BatchSize.ToString();
            XmlNode ndListItems = lists.GetListItems(doclib, null, ndQuery, ndViewFields, batchsize, ndQueryOptions, "");
            XmlNamespaceManager ns = new XmlNamespaceManager(ndListItems.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNodeList ndRows = ndListItems.SelectNodes(".//z:row", ns);
            List<SPDoclibItem> items = new List<SPDoclibItem>();
            string docliburlpath = new Uri(location.SiteUrl).PathAndQuery;
            if (ndRows.Count == 0) return null;
            return ndRows[0].NodeToDoclibItem();

        }
        public byte[] DownloadFile(string urlAbsolute)
        {
            LogVerboseMessage("DownloadFile,{0}", urlAbsolute);
            System.Net.WebClient cl = new System.Net.WebClient();
            cl.Credentials = System.Net.CredentialCache.DefaultCredentials;
            return cl.DownloadData(urlAbsolute);
        }
        public int RetryCount
        {
            get
            {
                throw new NotImplementedException();
            }
            set
            {
                throw new NotImplementedException();
            }
        }
        public int BatchSize
        {
            get
            {
                return _batchsize;
            }
            set
            {
                _batchsize = value;
            }
        }

        public SPDoclibItem[] GetAllItemsInFolder(SPDoclibLocation location, int startid, string rootfolder)
        {
            LogVerboseMessage("GetAllItemsInFolder,{0},{1}", startid, rootfolder);
            string weburl = location.SiteUrl;
            string doclib = location.DoclibName;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlDocument xmlDoc = new System.Xml.XmlDocument();

            XmlNode ndQuery = xmlDoc.CreateNode(XmlNodeType.Element, "Query", "");
            XmlNode ndViewFields = xmlDoc.CreateNode(XmlNodeType.Element, "ViewFields", "");
            XmlNode ndQueryOptions =
                xmlDoc.CreateNode(XmlNodeType.Element, "QueryOptions", "");
            //<Folder></Folder>
            ///
            /// Create the folder location
            ///
            string folderSearch = location.SiteUrl;
            if (folderSearch.EndsWith("/")) folderSearch =folderSearch.Substring(0,folderSearch.Length-1);
            if (string.IsNullOrEmpty(rootfolder))
            {

            }
            else
            {
                rootfolder = rootfolder.Trim();
                if (rootfolder.StartsWith("/")) rootfolder = rootfolder.Substring(1);
                if (rootfolder.EndsWith("/")) rootfolder=rootfolder.Substring(0,rootfolder.Length-1);
                folderSearch=folderSearch+"/"+ location.DoclibName +"/" + rootfolder;
            }
            string queryOptions=
                "<Folder>{0}</Folder>" +
                "<IncludeMandatoryColumns>FALSE</IncludeMandatoryColumns>" +
                "<DateInUtc>TRUE</DateInUtc>";
            queryOptions = string.Format(queryOptions, folderSearch);
            ndQueryOptions.InnerXml = queryOptions;

            ndViewFields.InnerXml = (new string[] { 
                                Constants.FIELD_CREATED, Constants.FIELD_MODIFIED , Constants.FIELD_VERSION , 
                                Constants.FIELD_VERSIONSTRING , Constants.FIELD_ITEMTYPE,Constants.FIELD_FILESIZE,
                                Constants.FIELD_TITLE,Constants.FIELD_ENCODEDABSURL}).ConvertToViewFields();
            ndQuery.InnerXml = string.Format("<Where><Gt><FieldRef Name='ID'/><Value Type='Counter'>{0}</Value></Gt>" + "</Where>", startid);

            string batchsize = this.BatchSize.ToString();
            XmlNode ndListItems = lists.GetListItems(doclib, null, ndQuery, ndViewFields, batchsize, ndQueryOptions, "");
            XmlNamespaceManager ns = new XmlNamespaceManager(ndListItems.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNodeList ndRows = ndListItems.SelectNodes(".//z:row", ns);
            List<SPDoclibItem> items = new List<SPDoclibItem>();
            string docliburlpath = new Uri(location.SiteUrl).PathAndQuery;
            foreach (XmlNode ndRow in ndRows)
            {
                string outerxml = ndRow.OuterXml;
                SPDoclibItem item = null;
                try
                {
                    item = ndRow.NodeToDoclibItem();
                    item.RelativeUrl = item.ServerRelativeUrl.Substring(docliburlpath.Length);
                    items.Add(item);
                }
                catch (Exception ex)
                {
                    LogError("Error while parsing SP List Item:");
                    LogError(outerxml);
                    LogError(ex.ToString());
                }
            }
            return items.ToArray();
        }

        /// <summary>
        /// Recursively copies all files and folders from source to destination.
        /// The callback function is used for verbose logging by Powersehll cmdlet.
        /// </summary>
        /// <param name="source"></param>
        /// <param name="dest"></param>
        /// <param name="LogMessage"></param>
        public SPReplicationResults Replicate(SPDoclibLocation source, SPDoclibLocation dest, bool overwrite)
        {
            Action<SPDoclibLocation, string> fnRecursiveCallBackCreateFoldersFiles = null;
            //int totalFilesCreated = 0, totalFoldersCreated = 0, totalFilesSkipped = 0, totalFoldersSkipped = 0;
            List<SPDoclibItem> filesCreated = new List<SPDoclibItem>();
            List<SPDoclibItem> filesSkipped = new List<SPDoclibItem>();
            List<SPDoclibItem> foldersCreated = new List<SPDoclibItem>();
            List<SPDoclibItem> foldersSkipped = new List<SPDoclibItem>();

            fnRecursiveCallBackCreateFoldersFiles = delegate(SPDoclibLocation srclocn, string folderpath)
            {
                LogVerboseMessage(FormatMessage("Searching for all items in:{0},{1},{2}",srclocn.SiteUrl,srclocn.DoclibName,folderpath));
                SPDoclibItem[] allitemsinthisSourceFolder = this.GetAllItemsInFolder(srclocn, 0, folderpath);
                LogVerboseMessage(FormatMessage( "Found {0} items in Source",allitemsinthisSourceFolder.Count()) );
                LogVerboseMessage(FormatMessage("Searching for all items in:{0},{1},{2}", dest.SiteUrl, dest.DoclibName, folderpath));
                SPDoclibItem[] allitemsinthisDestFolder = this.GetAllItemsInFolder(dest, 0, folderpath);
                LogVerboseMessage(FormatMessage( "Found {0} items in Destination",allitemsinthisDestFolder.Count()) );
                ///
                /// Verify that all sub-folders exist on destination
                ///
                SPDoclibItem[] allSourcefolders = allitemsinthisSourceFolder.Where(f => (f.IsFolder == true)).ToArray();
                if (allitemsinthisSourceFolder.Count() ==0)
                {
                    LogVerboseMessage("No folders found in Source");
                }
                else
                {
                    LogVerboseMessage(FormatMessage("{0} folders found in Source",allSourcefolders.Count()));
                    foreach(SPDoclibItem folderSource in allSourcefolders)
                    {
                        SPDoclibItem folderOnDest = allitemsinthisDestFolder.FirstOrDefault(it => (it.IsFolder == true && it.Name.Equals(folderSource.Name, StringComparison.CurrentCultureIgnoreCase)));
                        if (folderOnDest != null)
                        {
                            LogVerboseMessage(FormatMessage("'{0}\\{1}' exists in Destination.Skipping",folderpath,folderSource.Name));
                            foldersSkipped.Add(folderSource);
                            continue;
                        }
                        else
                        {
                            //needs creation
                            LogVerboseMessage(FormatMessage("Folder '{0}\\{1}' needs creation. Creating this now",folderpath,folderSource.Name));
                            SPDoclibItem newfolder=this.CreateFolder(dest, folderSource.Name, folderpath);
                            LogVerboseMessage(FormatMessage("Folder '{0}\\{1}' created",folderpath,folderSource.Name));
                            foldersCreated.Add(newfolder);
                        }
                    }
                }
                ///
                /// Verify that all files in samplefolder exist on destination
                ///
                LogVerboseMessage(FormatMessage("Searching for files in Source:'{0}\\{1}' ",srclocn.DoclibName,folderpath));
                SPDoclibItem[] allFilesinSourcefolder = allitemsinthisSourceFolder.Where(f => (f.IsFolder == false)).ToArray();
                LogVerboseMessage(FormatMessage("Found {2} files in Source:'{0}\\{1}' ",srclocn.DoclibName,folderpath,allFilesinSourcefolder.Count()));
                foreach (SPDoclibItem fileSource in allFilesinSourcefolder)
                {
                    SPDoclibItem fileOnDest = allitemsinthisDestFolder.FirstOrDefault(it => (it.IsFolder == false && it.Name.Equals(fileSource.Name, StringComparison.CurrentCultureIgnoreCase)));
                    if (fileOnDest != null)
                    {
                        //file with similar name exists, check if out of date only when user has asked specifically to overwrite
                        if (overwrite==true)
                        {
                            bool isstale=(fileOnDest.Modified < fileSource.Modified);
                            LogVerboseMessage(FormatMessage("File '{0}\\{1}' exists in Destination. Is it stale?{2}",folderpath,fileOnDest.Name,isstale));
                            if (isstale)
                            {
                                //File is out of date
                                LogVerboseMessage(FormatMessage("Copying file :{0}\\{1}\\{2} because source was more recent",srclocn.DoclibName,folderpath,fileSource.Name)); 
                                byte[] contents=this.DownloadFile(fileSource.AbsoluteUrl);
                                this.UploadFile(contents,dest,fileSource.Name,folderpath,fileSource.Title);
                                filesCreated.Add(fileSource);
                            }
                            else
                            {
                                //File is upto date. Do nothing
                                LogVerboseMessage(FormatMessage("Not copying file :{0}\\{1}\\{2} because destination is in sync", srclocn.DoclibName, folderpath, fileSource.Name));
                                filesSkipped.Add(fileSource);
                                continue;
                            }
                        }
                        else
                        {
                            //User does not want any onverwriting
                            LogVerboseMessage(FormatMessage("Not copying file :{0}\\{1}\\{2} because user does not want overwriting", srclocn.DoclibName, folderpath, fileSource.Name));
                            filesSkipped.Add(fileSource);
                            continue;
                        }
                    }
                    else
                    {
                        //file does not exist on destination, so upload the file
                        LogVerboseMessage(FormatMessage("Copying file :{0}\\{1}\\{2} because it does not exist", srclocn.DoclibName, folderpath, fileSource.Name)); 
                        byte[] bSourceFile = this.DownloadFile(fileSource.AbsoluteUrl);
                        this.UploadFile(bSourceFile, dest, fileSource.Name, folderpath, fileSource.Title);
                        LogVerboseMessage("Copying file complete.");
                        filesCreated.Add(fileSource);
                    }
                }
                foreach(SPDoclibItem folderSource in allSourcefolders)
                {
                fnRecursiveCallBackCreateFoldersFiles(source, folderpath+"/"+folderSource.Name);
                }
            };
            fnRecursiveCallBackCreateFoldersFiles(source,"");//begin from root
            LogVerboseMessage(FormatMessage("{0} files and {1} folders were created", filesCreated.Count(), foldersCreated.Count()));
            LogVerboseMessage(FormatMessage("{0} files and {1} folders were skipped", filesSkipped.Count(), foldersSkipped.Count()));
            LogVerboseMessage("***************Complete**********");
            SPReplicationResults result = new SPReplicationResults();
            result.FilesCopied = filesCreated.ToArray();
            result.FilesSkipped = filesSkipped.ToArray();
            result.FoldersCreated = foldersCreated.ToArray();
            result.FoldersSkipped = foldersSkipped.ToArray();
            return result;
        }
        /// <summary>
        /// Internal wrapper over the WriteVerbose of Powershell. To facilitate stubbing when unit testing
        /// </summary>
        /// <param name="format"></param>
        /// <param name="parameters"></param>
        private void LogVerboseMessage(string format,params object[] parameters)
        {
            if (_verboselogger!=null)
            {
                string message = FormatMessage(format, parameters);
                _verboselogger(message);
            }
        }
        /// <summary>
        /// Writes the error to the PowerShell. While unit testing the output is redirected to a stub to facilitate unit testing
        /// </summary>
        /// <param name="format"></param>
        /// <param name="parameters"></param>
        private void LogError(string format, params object[] parameters)
        {
            if (_errorLogger != null)
            {
                string message = FormatMessage(format, parameters);
                _errorLogger(message);
            }
            else
            {
                LogVerboseMessage(format, parameters);//as a fall back mechanism
            }
        }
        private string FormatMessage(string format, params object[] parameters)
        {
            return string.Format(format, parameters);
        }


        public entity.SPList GetList(SPDoclibLocation location)
        {
            LogVerboseMessage("GetList,{0},{1}", location.SiteUrl, location.DoclibName);
            string weburl = location.SiteUrl;
            string doclib = location.DoclibName;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlNode ndListInfo= lists.GetList(location.DoclibName);
            LogVerboseMessage("Web service invocation complete");
            entity.SPList listinfo = ndListInfo.NodeToSPList();
            return listinfo;
        }


        public entity.SPListItem[] GetListItems(SPDoclibLocation location, int startid, int maxitems, string[] viewfields, bool dump, string file)
        {
            LogVerboseMessage("GetListItems,{0},{1}", location.SiteUrl, location.DoclibName);
            string weburl = location.SiteUrl;
            string doclib = location.DoclibName;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlDocument xmlDoc = new System.Xml.XmlDocument();

            XmlNode ndQuery = xmlDoc.CreateNode(XmlNodeType.Element, "Query", "");
            XmlNode ndViewFields = xmlDoc.CreateNode(XmlNodeType.Element, "ViewFields", "");
            XmlNode ndQueryOptions =
                xmlDoc.CreateNode(XmlNodeType.Element, "QueryOptions", "");
            //<Folder></Folder>
            ndQueryOptions.InnerXml =
                "<ViewAttributes Scope='RecursiveAll'/>" +
                "<IncludeMandatoryColumns>FALSE</IncludeMandatoryColumns>" +
                "<DateInUtc>TRUE</DateInUtc>";
            List<string> fieldsToFetch = new List<string>();
            fieldsToFetch.AddRange(new string[] { 
                                Constants.FIELD_CREATED, Constants.FIELD_MODIFIED , Constants.FIELD_VERSION , 
                                Constants.FIELD_VERSIONSTRING , Constants.FIELD_ITEMTYPE,Constants.FIELD_FILESIZE,
                                Constants.FIELD_TITLE,Constants.FIELD_ENCODEDABSURL , Constants.FIELD_FILELEAFREF});
            if (viewfields != null) fieldsToFetch.AddRange(viewfields);
            ndViewFields.InnerXml = (fieldsToFetch.ToArray()).ConvertToViewFields();
            ndQuery.InnerXml = string.Format("<Where><Gt><FieldRef Name='ID'/><Value Type='Counter'>{0}</Value></Gt>" + "</Where>", startid);

            string batchsize = maxitems.ToString();
            XmlNode ndListItems = lists.GetListItems(doclib, null, ndQuery, ndViewFields, batchsize, ndQueryOptions, "");
            XmlNamespaceManager ns = new XmlNamespaceManager(ndListItems.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNodeList ndRows = ndListItems.SelectNodes(".//z:row", ns);
            if (dump)
            {
                try
                {
                    LogVerboseMessage("Dumping XML output to file:{0}", file);
                    System.IO.File.WriteAllText(file,ndListItems.OuterXml);
                }
                catch (Exception ex)
                {
                    LogError("Error while dumping web service output to file:{0}\r\n{1}", file, ex.ToString());
                }
            }
            List<SPListItem> items = new List<SPListItem>();
            string docliburlpath = new Uri(location.SiteUrl).PathAndQuery;
            foreach (XmlNode ndRow in ndRows)
            {
                string outerxml = ndRow.OuterXml;
                SPListItem item = new SPListItem();
                try
                {
                    foreach (string field in fieldsToFetch)
                    {
                        string webfieldname="ows_" + field;
                        try
                        {
                        string fieldValue = ndRow.Attributes[webfieldname].Value;
                        item.FieldValues.Add(field, fieldValue);
                        }
                        catch
                        {
                            //do this defensively because a specified field may not be available
                            if (!item.FieldValues.ContainsKey(field))
                            {
                                item.FieldValues.Add(field, null);
                            }
                        }
                    }
                    item.SPID=Int32.Parse(ndRow.Attributes["ows_ID"].Value);
                    string fileleafref = ndRow.Attributes["ows_FileLeafRef"].Value;
                    string name = fileleafref.Replace(item.SPID + ";#","");
                    item.Name = name;
                    items.Add(item);
                }
                catch (Exception ex)
                {
                    LogError("Error while parsing SP List Item:");
                    LogError(outerxml);
                    LogError(ex.ToString());
                }
            }
            return items.ToArray();
        }
        public SPListItem[] GetListItemsEx(
                SPDoclibLocation location, int maxitems, string[] viewfields, string caml, 
                string viewname, bool utcDates, string folder, bool includemandatorycolumns, 
                bool isrecursive, bool dump, string file)
        {
            LogVerboseMessage("GetListItemsEx,{0},{1}", location.SiteUrl, location.DoclibName);
            string weburl = location.SiteUrl;
            string doclib = location.DoclibName;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlDocument xmlDoc = new System.Xml.XmlDocument();

            XmlNode ndQuery = xmlDoc.CreateNode(XmlNodeType.Element, "Query", "");
            XmlNode ndViewFields = xmlDoc.CreateNode(XmlNodeType.Element, "ViewFields", "");
            XmlNode ndQueryOptions =
                xmlDoc.CreateNode(XmlNodeType.Element, "QueryOptions", "");
            //<Folder></Folder>
            ndQueryOptions.InnerXml = "";

            StringBuilder sbQueryOptions = new StringBuilder();
            if (isrecursive)
            {
                sbQueryOptions.Append("<ViewAttributes Scope='RecursiveAll'/>");
            }
            string sIncludeMandatory = (includemandatorycolumns) ? "TRUE" : "FALSE";
            sbQueryOptions.AppendFormat("<IncludeMandatoryColumns>{0}</IncludeMandatoryColumns>",sIncludeMandatory);

            string sUctDates = (utcDates) ? "TRUE" : "FALSE";
            sbQueryOptions.AppendFormat("<DateInUtc>{0}</DateInUtc>", sUctDates);

            if (!string.IsNullOrEmpty(folder))
            {
                if (folder.StartsWith("/"))
                {
                    folder = folder.Substring(1);//Remove any leading slashes. SP will not accept.
                }
                sbQueryOptions.AppendFormat("<Folder>{0}</Folder>", folder);
            }
            ndQueryOptions.InnerXml = sbQueryOptions.ToString();

            List<string> fieldsToFetch = new List<string>();
            fieldsToFetch.AddRange(new string[] { 
                                Constants.FIELD_CREATED, Constants.FIELD_MODIFIED , Constants.FIELD_VERSION , 
                                Constants.FIELD_VERSIONSTRING , Constants.FIELD_ITEMTYPE,Constants.FIELD_FILESIZE,
                                Constants.FIELD_TITLE,Constants.FIELD_ENCODEDABSURL , Constants.FIELD_FILELEAFREF});
            if (viewfields != null) fieldsToFetch.AddRange(viewfields);
            ndViewFields.InnerXml = (fieldsToFetch.ToArray()).ConvertToViewFields();

            string camlQuery = "";
            if (!string.IsNullOrEmpty(caml))
            {
                camlQuery = caml;
                ndQuery.InnerXml = camlQuery;
            }
            else
            {
                ndQuery = null;
            }
            string batchsize = maxitems.ToString();
            XmlNode ndListItems = lists.GetListItems(doclib, viewname, ndQuery, ndViewFields, batchsize, ndQueryOptions, "");
            XmlNamespaceManager ns = new XmlNamespaceManager(ndListItems.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNodeList ndRows = ndListItems.SelectNodes(".//z:row", ns);
            if (dump)
            {
                try
                {
                    LogVerboseMessage("Dumping XML output to file:{0}", file);
                    System.IO.File.WriteAllText(file, ndListItems.OuterXml);
                }
                catch (Exception ex)
                {
                    LogError("Error while dumping web service output to file:{0}\r\n{1}", file, ex.ToString());
                }
            }
            List<SPListItem> items = new List<SPListItem>();
            string docliburlpath = new Uri(location.SiteUrl).PathAndQuery;
            foreach (XmlNode ndRow in ndRows)
            {
                string outerxml = ndRow.OuterXml;
                SPListItem item = new SPListItem();
                try
                {
                    foreach (string field in fieldsToFetch)
                    {
                        string webfieldname = "ows_" + field;
                        try
                        {
                            string fieldValue = ndRow.Attributes[webfieldname].Value;
                            item.FieldValues.Add(field, fieldValue);
                        }
                        catch
                        {
                            //do this defensively because a specified field may not be available
                            if (!item.FieldValues.ContainsKey(field))
                            {
                                item.FieldValues.Add(field, null);
                            }
                        }
                    }
                    item.SPID = Int32.Parse(ndRow.Attributes["ows_ID"].Value);
                    string fileleafref = ndRow.Attributes["ows_FileLeafRef"].Value;
                    string name = fileleafref.Replace(item.SPID + ";#", "");
                    item.Name = name;
                    items.Add(item);
                }
                catch (Exception ex)
                {
                    LogError("Error while parsing SP List Item:");
                    LogError(outerxml);
                    LogError(ex.ToString());
                }
            }
            return items.ToArray();
        }

        public SPDoclibItem UploadFile(byte[] filebytes, SPWebShell.entity.SPList listInfo ,SPDoclibLocation doclibDestination, string fileName, string folderpath, string[] properties, object[] values)
        {
            LogVerboseMessage("UploadFile with metadata,{0},{1},{2}", doclibDestination.SiteUrl, folderpath, fileName);
            try
            {
                if (string.IsNullOrEmpty(fileName)) throw new ArgumentNullException("The name of the file cannot be blank");
                ///
                /// Concatenate the URL to the document library . Remember that a 'doclib' can exist under Lists and not neccessarily under the root!!!
                ///
                Uri userver=new Uri(doclibDestination.SiteUrl);
                string fullUrlToDocLibRoot = string.Format("{0}://{1}:{2}{3}", userver.Scheme, userver.Host, userver.Port, listInfo.NameValues["RootFolder"]);
                ////string urlFullNewItem = doclibDestination.SiteUrl.Trim();
                ////if (!urlFullNewItem.EndsWith("/")) urlFullNewItem = urlFullNewItem + "/";
                ////string urlRelative = "";
                ////if (string.IsNullOrWhiteSpace(folderpath))
                ////{
                ////    urlFullNewItem = urlFullNewItem + doclibDestination.DoclibName + "/" + fileName;
                ////}
                ////else
                ////{
                ////    urlRelative = folderpath;
                ////    if (urlRelative.StartsWith("/")) urlRelative = urlRelative.Substring(1);//eliminate leading slash
                ////    urlFullNewItem = urlFullNewItem + doclibDestination.DoclibName + "/" + urlRelative + "/" + fileName;
                ////}
                string urlFullNewItem = "";
                if (string.IsNullOrWhiteSpace(folderpath))
                {
                    urlFullNewItem = fullUrlToDocLibRoot + "/" + fileName;
                }
                else
                {
                    string urlRelative = folderpath;
                    if (urlRelative.StartsWith("/")) urlRelative = urlRelative.Substring(1);//eliminate leading slash
                    urlFullNewItem = fullUrlToDocLibRoot + "/" + urlRelative + "/" + fileName;
                }
                LogVerboseMessage("URL to document library", urlFullNewItem);
                WebRequest request = WebRequest.Create(urlFullNewItem);
                request.Credentials = CredentialCache.DefaultCredentials;
                request.Method = "PUT";
                using (Stream stream = request.GetRequestStream())
                {
                    stream.Write(filebytes, 0, filebytes.Length);
                }
                WebResponse response = null;
                response = request.GetResponse();
                ///
                /// Get the etag - this is the unique ID
                ///
                string key = "ETag";
                string etag = response.Headers.Get(key);
                Regex r = new Regex(@"\{.{1,}\}");//get the GUID
                Match m = r.Match(etag);
                string uniqueid = "";
                if (m.Success)
                {
                    uniqueid = m.ToString();
                }
                else
                {
                    throw new Exception(string.Format("Could not parse Etag after uploading document:{0}", urlFullNewItem));
                }
                LogVerboseMessage("UniqueID of uploaded document:{0}", uniqueid);
                Guid id = new Guid(uniqueid);
                SPDoclibItem newitem = this.GetItemUsingUniqueId(doclibDestination, id);//this works
                if ((properties == null) || (properties.Length == 0))
                {
                    LogVerboseMessage("Zero meta-data attributes specified. Skipping meta-data");
                }
                else
                {
                    LogVerboseMessage("{0} meta-data attributes specified. Updating meta-data",properties.Length);
                    this.UpdateMetaData(doclibDestination, newitem.SPID, properties, values);
                }
                //checkin the file
                string weburl = doclibDestination.SiteUrl;
                string doclib = doclibDestination.DoclibName;
                SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
                lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
                weburl = weburl.Trim();
                if (!weburl.EndsWith("/")) weburl = weburl + "/";
                lists.Url = weburl + "_vti_bin/lists.asmx";
                LogVerboseMessage("Going to checkin the file");
                bool retval=lists.CheckInFile(newitem.AbsoluteUrl,"","1");
                LogVerboseMessage("Checkin completed, returnvalue from CheckInFile method={0}",retval);
                //SPListItem newListItem = new SPListItem { SPID=newitem.SPID };
                return newitem;
            }
            catch (WebException wex)
            {
                if (wex.Response != null)
                {
                    using (var errorResponse = (HttpWebResponse)wex.Response)
                    {
                        StringBuilder sbHeaders = GetHeadersFromResponse(errorResponse);
                        using (var reader = new StreamReader(errorResponse.GetResponseStream()))
                        {
                            HttpStatusCode code = ((System.Net.HttpWebResponse)(wex.Response)).StatusCode;
                            string error = reader.ReadToEnd();
                            throw new Exception(string.Format("Error while uploading file:{0}\\{1}\r\n{2}\r\n{3}\r\n{4}", folderpath, fileName, code, error,sbHeaders.ToString()));
                        }
                    }
                }
            }
            return null;
        }
        /// <summary>
        /// Extracts some useful information from the headers when an error occurs
        /// </summary>
        /// <param name="resp"></param>
        /// <returns></returns>
        private StringBuilder GetHeadersFromResponse(WebResponse resp)
        {
            string[] keystolookfor = { "X-MSDAVEXT_Error"  };
            StringBuilder sbResponse = new StringBuilder();
            try
            {
                foreach (string keyHeader in resp.Headers.AllKeys)
                {
                    bool containskey = keystolookfor.Any(k => (k.Equals(keyHeader, StringComparison.CurrentCultureIgnoreCase)));
                    if (containskey)
                    {
                        string value = resp.Headers[keyHeader];
                        string valuedecoded=System.Web.HttpUtility.UrlDecode(value);
                        sbResponse.AppendFormat("{0}={1}", keyHeader, valuedecoded);
                    }
                }
            }
            catch (Exception ex)
            {
                sbResponse.Append(ex.ToString());
            }
            return sbResponse;
        }
        public void UpdateMetaData(SPDoclibLocation list, int itemid, string[] properties, object[] values)
        {
            LogVerboseMessage("Updating the meta-data of the uploaded item to '{0}'", itemid);
            if (properties == null) throw new Exception("No properties found");
            if (properties.Length == 0) throw new Exception("No properties found");
            if (values == null) throw new Exception("No values found");
            if (values.Length == 0) throw new Exception("No values found");
            if (properties.Length!=values.Length) throw new Exception("The count of property names and their values do not match");
            LogVerboseMessage("Displaying property names");
            foreach (string prop in properties)
            {
                LogVerboseMessage("\t{0}",prop);
            }
            LogVerboseMessage("Displaying property values");
            foreach (Object value in values)
            {
                LogVerboseMessage("\t{0}", value);
            }

            string weburl = list.SiteUrl;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlNode ndResults = null;
            string xmlbatch = string.Format(
                "<Batch><Method ID='1' Cmd='Update' ><Field Name='ID'>{0}</Field></Method></Batch>",itemid);
            //xmlbatch = string.Format(xmlbatch,itemid);
            //<Field Name='ID'>{0}</Field><Field Name='Title'>{1}</Field>
            XmlDocument docBatch = new XmlDocument();
            docBatch.LoadXml(xmlbatch);
            XmlNode ndMethod = docBatch.SelectSingleNode(".//Method");
            int index=0;
            foreach (string property in properties)
            {
                //XmlDocument docField = new XmlDocument();
                XmlElement  docField = docBatch.CreateElement("Field");
                //docField.LoadXml("<Field ></Field>");
                XmlAttribute name = docBatch.CreateAttribute("Name");
                name.Value = property;
                docField.InnerText = string.Format("{0}",values[index]);
                docField.Attributes.Append(name);
                ndMethod.AppendChild(docField);
                index++;
            }
            ndResults = lists.UpdateListItems(list.DoclibName, docBatch);
            string error="";
            bool retval = ndResults.IsSPError(out error);
            if (retval) throw new Exception(error);
        }


        public SPUser[] GetSiteUsers(string urlSite)
        {
            string url2 = this.GetContainerWebFromDocLibUrl(urlSite);
            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            XmlNode ndAllUsersResponse = client.GetUserCollectionFromSite();//GetAllUserCollectionFromWeb();
            XmlNamespaceManager ns = new XmlNamespaceManager(ndAllUsersResponse.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/directory/");
            XmlNodeList ndUsers = ndAllUsersResponse.SelectNodes(".//z:User", ns);
            List<SPUser> allusers = new List<SPUser>();
            foreach (XmlNode nduser in ndUsers)
            {
                SPUser user = XmlToUser(nduser);
                allusers.Add(user);
            }
            var qrySorted = from u in allusers orderby u.ID ascending select u;
            return qrySorted.ToArray();
        }
        SPUser XmlToUser(XmlNode nduser)
        {
            SPUser user = new SPUser();
            string sID = nduser.Attributes["ID"].Value;
            user.ID = Int32.Parse(sID);

            string sName = nduser.Attributes["Name"].Value;
            user.Name = string.IsNullOrEmpty(sName) ? "" : sName;

            string sSID = nduser.Attributes["Sid"].Value;
            user.Sid = string.IsNullOrEmpty(sSID) ? "" : sSID;

            string sLogin = nduser.Attributes["LoginName"].Value;
            user.LoginName = string.IsNullOrEmpty(sLogin) ? "" : sLogin;

            string sEmail = nduser.Attributes["Email"].Value;
            user.Email = string.IsNullOrEmpty(sEmail) ? "" : sEmail;

            string sNotes = nduser.Attributes["Notes"].Value;
            user.Notes = string.IsNullOrEmpty(sNotes) ? "" : sNotes;

            string sIsAdmin = nduser.Attributes["IsSiteAdmin"].Value;
            user.IsSiteAdmin = Boolean.Parse(sIsAdmin);

            string sIsDomainGroup = nduser.Attributes["IsDomainGroup"].Value;
            user.IsDomainGroup = Boolean.Parse(sIsDomainGroup);

            return user;
        }
        public SPGroup[] GetSiteGroups(string urlSite)
        {
            SPWebShell.SPWebs.Webs clWebs = new SPWebShell.SPWebs.Webs();
            clWebs.Credentials = System.Net.CredentialCache.DefaultCredentials;
            string url1 = this.GetContainerWebFromDocLibUrl(urlSite);
            string url2 = url1;// clWebs.WebUrlFromPageUrl(url1);

            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            XmlNode ndGroupsResponse=client.GetGroupCollectionFromSite();
            XmlNamespaceManager ns = new XmlNamespaceManager(ndGroupsResponse.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/directory/");
            XmlNodeList ndGroups = ndGroupsResponse.SelectNodes(".//z:Group",ns);
            List<SPGroup> allgroups = new List<SPGroup>();
            foreach (XmlNode ndGroup in ndGroups)
            {
                SPGroup group = XmlToSPGroup(ndGroup);
                allgroups.Add(group);

            }
            var qrySorted = from u in allgroups orderby u.ID ascending select u;
            SPUser[] allusers = this.GetSiteUsers(urlSite);
            foreach (SPGroup g in qrySorted)
            {
                int ownerid = g.OwnerID;
                if (g.OwnerIsUser)
                {
                    SPUser uOwner=allusers.FirstOrDefault(u=>(u.ID==ownerid));
                    g.OwnerName = uOwner.Name;
                }
                else
                {
                    SPGroup gOwner = allgroups.FirstOrDefault(gr => (gr.ID == ownerid));
                    g.OwnerName = gOwner.Name;
                }
            }
            return qrySorted.ToArray();
        }
        SPGroup XmlToSPGroup(XmlNode ndGroup)
        {
            SPGroup group = new SPGroup();
            string sID = ndGroup.Attributes["ID"].Value;
            group.ID = Int32.Parse(sID);

            string sName = ndGroup.Attributes["Name"].Value;
            group.Name = string.IsNullOrEmpty(sName) ? "" : sName;

            string sDescription = ndGroup.Attributes["Description"].Value;
            group.Description = string.IsNullOrEmpty(sDescription) ? "" : sDescription;

            string sOwnerID = ndGroup.Attributes["OwnerID"].Value;
            group.OwnerID = string.IsNullOrEmpty(sOwnerID) ? 0 : Int32.Parse(sOwnerID);

            string sOwnerIsUser = ndGroup.Attributes["OwnerIsUser"].Value;
            group.OwnerIsUser = Boolean.Parse(sOwnerIsUser);
            return group;
        }

        //public bool CheckinFile(string urlAbsolute)
        //{
        //    string weburl = location.SiteUrl;
        //    string doclib = location.DoclibName;
        //    SPLists.Lists lists = new SPLists.Lists();
        //    lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
        //    weburl = weburl.Trim();
        //    if (!weburl.EndsWith("/")) weburl = weburl + "/";
        //    lists.Url = weburl + "_vti_bin/lists.asmx";
        //    throw new NotImplementedException();
        //}

        //public bool CheckinFile(SPDoclibLocation location, int spid)
        //{
        //    throw new NotImplementedException();
        //}


        public Guid PutAnyDocument(byte[] filebytes, string urlAbsolute)
        {
            LogVerboseMessage("method PutAnyDocument, url:{0}, content length:{1}", urlAbsolute, filebytes.Length);
            WebRequest request = WebRequest.Create(urlAbsolute);
            request.Credentials = CredentialCache.DefaultCredentials;
            request.Method = "PUT";
            LogVerboseMessage("Before calling submitted web request ");
            using (Stream stream = request.GetRequestStream())
            {
                stream.Write(filebytes, 0, filebytes.Length);
            }
            WebResponse response = null;
            response = request.GetResponse();
            LogVerboseMessage("Response obtained");
            LogVerboseMessage("Looking for etag in the response headers");
            string key = "ETag";
            string etag = response.Headers.Get(key);
            if ((etag == null) || (string.IsNullOrEmpty(etag)))
            {
                LogVerboseMessage("etag header was empty");
                return Guid.Empty;
            }
            Regex r = new Regex(@"\{.{1,}\}");//get the GUID
            Match m = r.Match(etag);
            string uniqueid = "";
            if (m.Success)
            {
                LogVerboseMessage("etag foundin header:{0}", etag);
                uniqueid = m.ToString();
                return new Guid(uniqueid);
            }
            else
            {
                LogError("Could not parse Etag after uploading document:{0}", urlAbsolute);
                return Guid.Empty;
            }
        }

        public SPWeb[] GetWebs(string urlSite)
        {
            SPWebShell.SPWebs.Webs client = new SPWebShell.SPWebs.Webs();
            string url2 = urlSite;// this.GetContainerWebFromDocLibUrl(urlSite);//Not decoding container site - problem with recursion
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/Webs.asmx";
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            XmlNode ndResults=client.GetWebCollection();
            return XmlToSpWebs(ndResults);
        }
        /// <summary>
        /// Used for processing results from GetWeb or GetWebCollection.
        /// The XML differs slightly and hence the if-else
        /// </summary>
        /// <param name="ndResults"></param>
        /// <returns></returns>
        SPWeb[] XmlToSpWebs(XmlNode ndResults)
        {
            XmlNamespaceManager ns = new XmlNamespaceManager(ndResults.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/");
            List<SPWeb> webs = new List<SPWeb>();
            Func<XmlNode, SPWeb> processnode = delegate(XmlNode ndWeb)
            {
                SPWeb web = new SPWeb();
                web.Title = ndWeb.Attributes["Title"].Value;
                web.Url = ndWeb.Attributes["Url"].Value;
                return web;

            };
            if (ndResults.Name=="Web")
            {
                //When processing result from GetWeb
                SPWeb web = processnode(ndResults);
                webs.Add(web);
            }
            else if (ndResults.Name=="Webs")
            {
                //When processing results from GetWebCollection
                XmlNodeList ndWebs = ndResults.SelectNodes("//z:Web", ns);
                foreach (XmlNode ndWeb in ndWebs)
                {
                    SPWeb web = processnode(ndWeb);
                    webs.Add(web);
                }
            }
            else
            {
                throw new NotImplementedException("");
            }
            return webs.ToArray();
        }

        public SPUser[] GetGroupMembers(string urlSite, string groupName)
        {
            string url2 = this.GetContainerWebFromDocLibUrl(urlSite);
            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            XmlNode ndAllUsersResponse = client.GetUserCollectionFromGroup(groupName);
            XmlNamespaceManager ns = new XmlNamespaceManager(ndAllUsersResponse.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/directory/");
            XmlNodeList ndUsers = ndAllUsersResponse.SelectNodes(".//z:User", ns);
            List<SPUser> allusers = new List<SPUser>();
            foreach (XmlNode nduser in ndUsers)
            {
                SPUser user = XmlToUser(nduser);
                allusers.Add(user);
            }
            var qrySorted = from u in allusers orderby u.ID ascending select u;
            return qrySorted.ToArray();            
        }


        public SPGroup[] GetGroupsFromUser(string urlSite, string userLogin)
        {
            LogVerboseMessage("Inside function GetGroupsFromUser:{0},{1}",urlSite,userLogin);
            string url1 = this.GetContainerWebFromDocLibUrl(urlSite);
            string url2 = url1;// clWebs.WebUrlFromPageUrl(url1);

            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            LogVerboseMessage("Before calling web method GetGroupCollectionFromUser");
            XmlNode ndGroupsResponse = client.GetGroupCollectionFromUser(userLogin);
            LogVerboseMessage("After calling web method GetGroupCollectionFromUser");
            XmlNamespaceManager ns = new XmlNamespaceManager(ndGroupsResponse.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/directory/");
            XmlNodeList ndGroups = ndGroupsResponse.SelectNodes(".//z:Group", ns);
            List<SPGroup> allgroupsforuser = new List<SPGroup>();
            foreach (XmlNode ndGroup in ndGroups)
            {
                SPGroup group = XmlToSPGroup(ndGroup);
                allgroupsforuser.Add(group);

            }
            //
            //Get all groups from Site - this is to resolve the owner information
            //
            LogVerboseMessage("Before calling GetGroupCollectionFromSite");
            XmlNode ndAllSiteGroupsResponse = client.GetGroupCollectionFromSite();
            LogVerboseMessage("After calling GetGroupCollectionFromSite");
            ns = new XmlNamespaceManager(ndAllSiteGroupsResponse.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/directory/");
            XmlNodeList ndAllSiteGroups = ndAllSiteGroupsResponse.SelectNodes(".//z:Group", ns);

            List<SPGroup> allSiteGroups = new List<SPGroup>();
            foreach (XmlNode ndGroup in ndAllSiteGroups)
            {
                SPGroup group = XmlToSPGroup(ndGroup);
                allSiteGroups.Add(group);
            }
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/directory/");
            var qrySorted = from u in allgroupsforuser orderby u.ID ascending select u;
            LogVerboseMessage("Before calling GetSiteUsers");
            SPUser[] allusers = this.GetSiteUsers(urlSite);
            LogVerboseMessage("After calling GetSiteUsers");
            foreach (SPGroup g in qrySorted)
            {
                int ownerid = g.OwnerID;
                if (g.OwnerIsUser)
                {
                    SPUser uOwner = allusers.FirstOrDefault(u => (u.ID == ownerid));
                    g.OwnerName = uOwner.Name;
                }
                else
                {
                    SPGroup gOwner = allSiteGroups.FirstOrDefault(gr => (gr.ID == ownerid));
                    g.OwnerName = gOwner.Name;
                }
            }
            return qrySorted.ToArray();     
        }
        public void AddUserToGroup(string urlSite, string userLogin,string group)
        {
            LogVerboseMessage("Inside function AddUserToGroup:{0},{1}", urlSite, userLogin);
            string url1 = this.GetContainerWebFromDocLibUrl(urlSite);
            string url2 = url1;// clWebs.WebUrlFromPageUrl(url1);

            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            LogVerboseMessage("Before calling web method AddUserToGroup");
            client.AddUserToGroup(group,"",userLogin,"","");
            LogVerboseMessage("After calling web method AddUserToGroup");            
        }
        public void RemoveUserFromGroup(string urlSite, string userLogin,string group)
        {
            LogVerboseMessage("Inside function AddUserToGroup:{0},{1},{2}", urlSite, userLogin,group);
            string url1 = this.GetContainerWebFromDocLibUrl(urlSite);
            string url2 = url1;// clWebs.WebUrlFromPageUrl(url1);

            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            LogVerboseMessage("Before calling RemoveUserFromGroup");
            client.RemoveUserFromGroup(group,userLogin);
            LogVerboseMessage("After calling RemoveUserFromGroup");            
        }


        public entity.SPList[] GetLists(string url)
        {
            string url2 = url;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            SPWebShell.SPLists.Lists client = new SPWebShell.SPLists.Lists();
            client.Url = url2 + "_vti_bin/Lists.asmx";
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            XmlNode ndResults=client.GetListCollection();
            XmlNamespaceManager ns = new XmlNamespaceManager(ndResults.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/");
            List<SPList> lists = new List<SPList>();
            XmlNodeList ndListNodes = ndResults.SelectNodes(".//z:List", ns);
            foreach (XmlNode ndList in ndListNodes)
            {
                string sID = ndList.Attributes["ID"].Value;
                string sDescrip = ndList.Attributes["Description"].Value;
                SPList list = new SPList(ndList);
                lists.Add(list);
            }
            return lists.ToArray();
        }


        public void RemoveUserFromSite(string urlSite, string userLogin)
        {
            LogVerboseMessage("Inside function RemoveUserFromSite:{0},{1}", urlSite, userLogin);
            string url1 = this.GetContainerWebFromDocLibUrl(urlSite);
            string url2 = url1;// clWebs.WebUrlFromPageUrl(url1);

            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            LogVerboseMessage("Before calling RemoveUserFromSite");
            client.RemoveUserFromSite( userLogin);
            LogVerboseMessage("After calling RemoveUserFromSite");                        
        }


        public void DeleteItem(SPDoclibLocation list, int itemid)
        {
            LogVerboseMessage("Deleting the list item with id:'{0}' from listname:'{1}'", itemid,list.DoclibName);
            LogVerboseMessage("Fetching this item ");
            SPListItem thisItem=this.GetListItem(list, itemid, 1, new string[] {"FileRef"}, false, "");
            if (thisItem == null) throw new Exception("No list item with the specified ID was found");
            string weburl = list.SiteUrl;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlNode ndResults = null;
            string absurl = (string)thisItem.FieldValues["EncodedAbsUrl"];
            string absUrlDecoded=System.Web.HttpUtility.UrlDecode(absurl);
            LogVerboseMessage("Decoded url:{0}", absUrlDecoded);
            LogVerboseMessage(absurl);
            string xmlbatch = string.Format(
                "<Batch><Method ID='1' Cmd='Delete' ><Field Name='ID'>{0}</Field><Field Name='FileRef'>{1}</Field></Method></Batch>", itemid, absUrlDecoded);
            //<Field Name='FileRef'>{1}</Field>
            XmlDocument docBatch = new XmlDocument();
            docBatch.LoadXml(xmlbatch);

            ndResults = lists.UpdateListItems(list.DoclibName, docBatch);
            string error = "";
            bool retval = ndResults.IsSPError(out error);
            if (retval) throw new Exception(error);

        }


        public SPListItem GetListItem(SPDoclibLocation location, int itemid, int maxitems, string[] viewfields, bool dump, string file)
        {
            LogVerboseMessage("GetListItem,{0},{1},{2}", location.SiteUrl, location.DoclibName,itemid);
            string weburl = location.SiteUrl;
            string doclib = location.DoclibName;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlDocument xmlDoc = new System.Xml.XmlDocument();

            XmlNode ndQuery = xmlDoc.CreateNode(XmlNodeType.Element, "Query", "");
            XmlNode ndViewFields = xmlDoc.CreateNode(XmlNodeType.Element, "ViewFields", "");
            XmlNode ndQueryOptions =
                xmlDoc.CreateNode(XmlNodeType.Element, "QueryOptions", "");
            //<Folder></Folder>
            ndQueryOptions.InnerXml =
                "<ViewAttributes Scope='RecursiveAll'/>" +
                "<IncludeMandatoryColumns>FALSE</IncludeMandatoryColumns>" +
                "<DateInUtc>TRUE</DateInUtc>";
            List<string> fieldsToFetch = new List<string>();
            fieldsToFetch.AddRange(new string[] { 
                                Constants.FIELD_CREATED, Constants.FIELD_MODIFIED , Constants.FIELD_VERSION , 
                                Constants.FIELD_VERSIONSTRING , Constants.FIELD_ITEMTYPE,Constants.FIELD_FILESIZE,
                                Constants.FIELD_TITLE,Constants.FIELD_ENCODEDABSURL , Constants.FIELD_FILELEAFREF});
            if (viewfields != null) fieldsToFetch.AddRange(viewfields);
            ndViewFields.InnerXml = (fieldsToFetch.ToArray()).ConvertToViewFields();
            ndQuery.InnerXml = string.Format("<Where><Eq><FieldRef Name='ID'/><Value Type='Counter'>{0}</Value></Eq>" + "</Where>", itemid);

            string batchsize = maxitems.ToString();
            XmlNode ndListItems = lists.GetListItems(doclib, null, ndQuery, ndViewFields, batchsize, ndQueryOptions, "");
            XmlNamespaceManager ns = new XmlNamespaceManager(ndListItems.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNodeList ndRows = ndListItems.SelectNodes(".//z:row", ns);
            if (dump)
            {
                try
                {
                    LogVerboseMessage("Dumping XML output to file:{0}", file);
                    System.IO.File.WriteAllText(file, ndListItems.OuterXml);
                }
                catch (Exception ex)
                {
                    LogError("Error while dumping web service output to file:{0}\r\n{1}", file, ex.ToString());
                }
            }
            List<SPListItem> items = new List<SPListItem>();
            string docliburlpath = new Uri(location.SiteUrl).PathAndQuery;
            foreach (XmlNode ndRow in ndRows)
            {
                string outerxml = ndRow.OuterXml;
                SPListItem item = new SPListItem();
                try
                {
                    foreach (string field in fieldsToFetch)
                    {
                        string webfieldname = "ows_" + field;
                        try
                        {
                            string fieldValue = ndRow.Attributes[webfieldname].Value;
                            item.FieldValues.Add(field, fieldValue);
                        }
                        catch
                        {
                            //do this defensively because a specified field may not be available
                            if (!item.FieldValues.ContainsKey(field))
                            {
                                item.FieldValues.Add(field, null);
                            }
                        }
                    }
                    item.SPID = Int32.Parse(ndRow.Attributes["ows_ID"].Value);
                    string fileleafref = ndRow.Attributes["ows_FileLeafRef"].Value;
                    string name = fileleafref.Replace(item.SPID + ";#", "");
                    item.Name = name;
                    items.Add(item);
                }
                catch (Exception ex)
                {
                    LogError("Error while parsing SP List Item:");
                    LogError(outerxml);
                    LogError(ex.ToString());
                }
            }
            return items.FirstOrDefault();
        }


        //public string ValidateWorkflow(string siteUrl,string contentsMarkup, string contentsRules, string contentsConfig , string flag)
        //{
        //    LogVerboseMessage("ValidateWorkflow:{0}", siteUrl);
        //    SPWebPartPages.WebPartPagesWebService webpartpages = new SPWebPartPages.WebPartPagesWebService();
        //    webpartpages.Credentials = System.Net.CredentialCache.DefaultCredentials;
        //    siteUrl = siteUrl.Trim();
        //    if (!siteUrl.EndsWith("/")) siteUrl = siteUrl + "/";
        //    webpartpages.Url = siteUrl + "_vti_bin/WebpartPages.asmx";
        //    string result=webpartpages.ValidateWorkflowMarkupAndCreateSupportObjects(contentsMarkup, contentsRules, contentsConfig, flag);
        //    return result;
        //}


        //public string AssociateWorkflow(string weburl, string configurl, string version)
        //{
        //    LogVerboseMessage("AssociateWorkflow:{0},configurl:{1},version:{2}", weburl,configurl,version);
        //    if (configurl.StartsWith("/"))
        //    {
        //        //remove leading /
        //        configurl = configurl.Substring(1);
        //    }
        //    if (!version.StartsWith("V" , StringComparison.CurrentCultureIgnoreCase))
        //    {
        //        version = "V" + version;
        //    }
        //    SPWebPartPages.WebPartPagesWebService webpartpages = new SPWebPartPages.WebPartPagesWebService();
        //    webpartpages.Credentials = System.Net.CredentialCache.DefaultCredentials;
        //    weburl = weburl.Trim();
        //    if (!weburl.EndsWith("/")) weburl = weburl + "/";
        //    webpartpages.Url = weburl + "_vti_bin/WebpartPages.asmx";
        //    string result = webpartpages.AssociateWorkflowMarkup(configurl, version);
        //    return result;
        //}


        public SPWeb GetWeb(string url)
        {
            SPWebShell.SPWebs.Webs client = new SPWebShell.SPWebs.Webs();
            string url2 = url;// this.GetContainerWebFromDocLibUrl(urlSite);//Not decoding container site - problem with recursion
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/Webs.asmx";
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            XmlNode ndResult=client.GetWeb(url);
            return XmlToSpWebs(ndResult)[0];
        }

        public SPPermission[] GetPermissions(string urlContainerSite, bool recursive, bool lists, string listname)
        {
            LogVerboseMessage("Begin GetPermissions,recursive:{0},display lists:{1},url:{2}", recursive, lists, urlContainerSite);
            if (lists)
            {
                LogVerboseMessage("Enumerating permissions on a specific List is not yet impelmented. Please specify a List name to view the permissions");
                throw new NotImplementedException("Enumerating permissions on a specific List is not yet impelmented. Please specify a List name to view the permissions");
            }
            if ((!string.IsNullOrWhiteSpace(listname)) && (recursive))
            {
                throw new Exception("If a List is specified then recursive should be set to False");
            }
            string url2 = this.GetContainerWebFromDocLibUrl(urlContainerSite);
            SPWebShell.SPPermissionsService.Permissions servPerms = new SPWebShell.SPPermissionsService.Permissions();
            servPerms.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            //
            //Get the current web. You do want to skip the current web.
            //
            SPWebShell.SPWebs.Webs srvWebs = new SPWebShell.SPWebs.Webs();
            srvWebs.Url = url2 + "_vti_bin/Webs.asmx";
            srvWebs.Credentials = System.Net.CredentialCache.DefaultCredentials;
            LogVerboseMessage("Fetching URL and title of current web");
            XmlNode ndThisWeb = srvWebs.GetWeb(".");
            LogVerboseMessage("URL and title of current web obtained");
            SPWeb[] thisweb = XmlToSpWebs(ndThisWeb);
            List<SPWeb> masterlist = new List<SPWeb>();
            masterlist.AddRange(thisweb);

            Action<string> fnRecurse = null;
            fnRecurse = delegate(string url)
            {
                LogVerboseMessage(string.Format("Enumerating websites under:{0}", url));
                if (!url.EndsWith("/")) url = url + "/";
                srvWebs.Url = url + "_vti_bin/Webs.asmx";
                XmlNode ndWebs = srvWebs.GetWebCollection();
                SPWeb[] webs = XmlToSpWebs(ndWebs);
                foreach (SPWeb web in webs)
                {
                    if (!web.Url.EndsWith("/")) web.Url = web.Url + "/";
                    masterlist.Add(web);
                    fnRecurse(web.Url);
                }
            };
            if (!string.IsNullOrWhiteSpace(listname))
            {
                //A list name has been specified. Get permissions for a specific List
                LogVerboseMessage("Finding permissions for the List:{0}", listname);
                servPerms.Url = url2 + "_vti_bin/Permissions.asmx";
                XmlNode ndPerms = servPerms.GetPermissionCollection(listname, "list");
                entity.SPPermission[] perms = XmlToPermissions(ndPerms);
                foreach (SPPermission perm in perms)
                {
                    perm.Url = url2;
                    perm.Name = listname;
                    perm.IsList = true;
                }
                return perms;
            }
            else
            {
                //Get Web permissions
                if (recursive)
                {
                    LogVerboseMessage("Finding permissions recursively for the Web");
                    fnRecurse(url2);
                }
                else
                {
                    LogVerboseMessage("Finding permissions non-recursively for the Web");
                }
                LogVerboseMessage("No of sites discovered:{0}", masterlist.Count());
                List<entity.SPPermission> allpermissions = new List<SPPermission>();
                foreach (SPWeb web in masterlist)
                {
                    string urlThisWeb = web.Url;
                    if (!urlThisWeb.EndsWith("/")) urlThisWeb = urlThisWeb + "/";
                    servPerms.Url = urlThisWeb + "_vti_bin/Permissions.asmx";
                    LogVerboseMessage("Interrogating permissions for :{0}", web.Url);
                    XmlNode ndPerms = servPerms.GetPermissionCollection(web.Title, "web");
                    entity.SPPermission[] perms = XmlToPermissions(ndPerms);
                    LogVerboseMessage("Found {0} permissions", perms.Count());
                    foreach (SPPermission perm in perms)
                    {
                        perm.Url = web.Url;
                        perm.Name = web.Title;
                        perm.IsList = false;
                    }
                    allpermissions.AddRange(perms);
                }
                LogVerboseMessage("Found {0} permissions in {1} sites", allpermissions.Count(), masterlist.Count());
                return allpermissions.ToArray();
            }
        }
        entity.SPPermission[] XmlToPermissions(XmlNode ndPermissions)
        {
            XmlNamespaceManager ns = new XmlNamespaceManager(ndPermissions.OwnerDocument.NameTable);
            ns.AddNamespace("z", "http://schemas.microsoft.com/sharepoint/soap/directory/");
            XmlNodeList nodes = ndPermissions.SelectNodes("//z:Permission" ,ns);
            List<entity.SPPermission> perms = new List<SPPermission>();
            foreach(XmlNode ndPermission in nodes)
            {
                SPPermission perm = new SPPermission();
                perms.Add(perm);
                string memberid = ndPermission.Attributes["MemberID"].Value;                
                string mask = ndPermission.Attributes["Mask"].Value;
                int iMask = Int32.Parse(mask);
                string memberisuser = ndPermission.Attributes["MemberIsUser"].Value;
                perm.IsUser = Boolean.Parse(memberisuser);
                string entitynameattribute = "";
                entitynameattribute = (perm.IsUser) ? "UserLogin" : "GroupName";

                perm.PrincipalName = ndPermission.Attributes[entitynameattribute].Value;//UserLogin
                perm.Mask = Extensions.IntToMask(iMask);               
            }
            return perms.ToArray();
        }



        public void RemovePermissions(string urlSite, string listname, string userlogin, string groupname)
        {
            LogVerboseMessage("Begin RemovePermissions,url:{0},listname:{1},userlogin:{2},groupname:{3}",urlSite , listname,userlogin,groupname);
            if (string.IsNullOrWhiteSpace(userlogin) && string.IsNullOrWhiteSpace(groupname))
            {
                throw new Exception("Both Group name and User should not be specified at the same time.");
            }
            SPWebShell.SPPermissionsService.Permissions servPerms = new SPWebShell.SPPermissionsService.Permissions();
            servPerms.Credentials = System.Net.CredentialCache.DefaultCredentials;
            urlSite = urlSite.Trim();
            if (!urlSite.EndsWith("/")) urlSite = urlSite + "/";
            string objtype = "List";
            if (string.IsNullOrWhiteSpace(listname))
            {
                objtype = "web";
            }
            string objectname = listname;
            if (string.IsNullOrWhiteSpace(listname))
            {
                SPWebShell.SPWebs.Webs srvWebs = new SPWebShell.SPWebs.Webs();
                srvWebs.Url = urlSite + "_vti_bin/Webs.asmx";
                srvWebs.Credentials = System.Net.CredentialCache.DefaultCredentials;
                XmlNode ndThisWeb = srvWebs.GetWeb(".");
                LogVerboseMessage("URL and title of current web obtained");
                SPWeb[] thisweb = XmlToSpWebs(ndThisWeb);
                objectname = thisweb[0].Title;
            }
            string permtype = "user";
            string identity = userlogin;
            if (string.IsNullOrWhiteSpace(userlogin))
            {
                permtype = "group";
                identity = groupname;
            }
            servPerms.Url = urlSite + "_vti_bin/Permissions.asmx";
            servPerms.RemovePermission(objectname, objtype, identity, permtype);
            LogVerboseMessage("End RemovePermissions");
        }
        public void AddPermissions(string urlSite, string listname, string userlogin, string groupname, entity.SPPermissionMask mask)
        {
            LogVerboseMessage("Begin AddPermissions,url:{0},listname:{1},userlogin:{2},groupname:{3},mask:{4}", urlSite, listname, userlogin, groupname,mask);
            if (string.IsNullOrWhiteSpace(userlogin) && string.IsNullOrWhiteSpace(groupname))
            {
                throw new Exception("Both Group name and User should not be specified at the same time.");
            }
            SPWebShell.SPPermissionsService.Permissions servPerms = new SPWebShell.SPPermissionsService.Permissions();
            servPerms.Credentials = System.Net.CredentialCache.DefaultCredentials;
            urlSite = urlSite.Trim();
            if (!urlSite.EndsWith("/")) urlSite = urlSite + "/";
            string objtype = "List";
            if (string.IsNullOrWhiteSpace(listname))
            {
                objtype = "web";
            }
            string objectname = listname;
            if (string.IsNullOrWhiteSpace(listname))
            {
                SPWebShell.SPWebs.Webs srvWebs = new SPWebShell.SPWebs.Webs();
                srvWebs.Url = urlSite + "_vti_bin/Webs.asmx";
                srvWebs.Credentials = System.Net.CredentialCache.DefaultCredentials;
                XmlNode ndThisWeb = srvWebs.GetWeb(".");
                LogVerboseMessage("URL and title of current web obtained");
                SPWeb[] thisweb = XmlToSpWebs(ndThisWeb);
                objectname = thisweb[0].Title;
            }
            string permtype = "user";
            string identity = userlogin;
            if (string.IsNullOrWhiteSpace(userlogin))
            {
                permtype = "group";
                identity = groupname;
            }
            servPerms.Url = urlSite + "_vti_bin/Permissions.asmx";
            servPerms.AddPermission(objectname, objtype, identity, permtype,(int)mask);
            LogVerboseMessage("End AddPermissions");
        }

        public void AddGroup(string urlSiteCollection, string groupName, string ownerLogin, string ownerGroup, string description)
        {
            LogVerboseMessage("Begin AddGroup,url:{0},ownerLogin:{1},ownerGroup:{2},groupName:{3}", urlSiteCollection, ownerLogin, ownerGroup, groupName);
            if (!string.IsNullOrWhiteSpace(ownerGroup) && !string.IsNullOrWhiteSpace(ownerLogin))
            {
                throw new Exception("Both owner login and owner group should not be specified simultaneously.");
            }
            if (string.IsNullOrWhiteSpace(ownerGroup) && string.IsNullOrWhiteSpace(ownerLogin))
            {
                throw new Exception("Either owner login or owner group should be specified .");
            }
            string url1 = this.GetContainerWebFromDocLibUrl(urlSiteCollection);
            string url2 = url1;// clWebs.WebUrlFromPageUrl(url1);

            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            LogVerboseMessage("Before calling web method AddGroup ");
            string owneridentifier="";
            string ownertype = "";
            if (string.IsNullOrEmpty(ownerLogin))
            {
                owneridentifier = ownerGroup;
                ownertype = "group";
            }
            else
            {
                owneridentifier = ownerLogin;
                ownertype = "user";
            }
            string defaultuser = string.Format("{0}\\{1}", Environment.GetEnvironmentVariable("USERDOMAIN"), Environment.GetEnvironmentVariable("USERNAME"));
            client.AddGroup(groupName, owneridentifier, ownertype, defaultuser, description);
            LogVerboseMessage("After calling web method AddGroup ");
            LogVerboseMessage("End");
        }


        public void RemoveGroup(string urlSiteCollection, string groupName)
        {
            LogVerboseMessage("Begin RemoveGroup,url:{0},groupName:{1}", urlSiteCollection,  groupName);
            string url1 = this.GetContainerWebFromDocLibUrl(urlSiteCollection);
            string url2 = url1;// clWebs.WebUrlFromPageUrl(url1);

            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            LogVerboseMessage("Before calling web method RemoveGroup ");
            client.RemoveGroup(groupName);
            LogVerboseMessage("After calling web method RemoveGroup ");
            LogVerboseMessage("End");
        }


        public SPListItem CreateListItem(SPDoclibLocation list, string[] properties, object[] values)
        {
            LogVerboseMessage("Creating a new list item '{0}'", list.DoclibName);
            if (properties == null)
            {
                properties = new string[] { };
            }
            if (values == null)
            {
                values = new string[] { };
            }
            if (properties.Length != values.Length) throw new Exception("The count of property names and their values do not match");
            LogVerboseMessage("Displaying property names");
            foreach (string prop in properties)
            {
                LogVerboseMessage("\t{0}", prop);
            }
            LogVerboseMessage("Displaying property values");
            foreach (Object value in values)
            {
                LogVerboseMessage("\t{0}", value);
            }

            string weburl = list.SiteUrl;
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            weburl = weburl.Trim();
            if (!weburl.EndsWith("/")) weburl = weburl + "/";
            lists.Url = weburl + "_vti_bin/lists.asmx";
            XmlNode ndResults = null;
            string xmlbatch = string.Format(
                "<Batch><Method ID='1' Cmd='New' ></Method></Batch>");
            XmlDocument docBatch = new XmlDocument();
            docBatch.LoadXml(xmlbatch);
            XmlNode ndMethod = docBatch.SelectSingleNode(".//Method");
            int index = 0;
            foreach (string property in properties)
            {
                //XmlDocument docField = new XmlDocument();
                XmlElement docField = docBatch.CreateElement("Field");
                //docField.LoadXml("<Field ></Field>");
                XmlAttribute name = docBatch.CreateAttribute("Name");
                name.Value = property;
                docField.InnerText = string.Format("{0}", values[index]);
                docField.Attributes.Append(name);
                ndMethod.AppendChild(docField);
                index++;
            }
            ndResults = lists.UpdateListItems(list.DoclibName, docBatch);
            string error = "";
            bool retval = ndResults.IsSPError(out error);
            if (retval) throw new Exception(error);
            XmlNamespaceManager ns = new XmlNamespaceManager(ndResults.OwnerDocument.NameTable);
            ns.AddNamespace("z", "#RowsetSchema");
            XmlNode ndRow=ndResults.SelectSingleNode(".//z:row",ns);
            string newid = ndRow.Attributes["ows_ID"].Value;
            int iId = Int32.Parse(newid);
            entity.SPListItem newitem = new SPListItem {  SPID=iId};
            return newitem;
        }


        public void UpdateUserInformation(string sitecollectionurl,string login, string displayname,string email,string notes)
        {
            string url2 = this.GetContainerWebFromDocLibUrl(sitecollectionurl);
            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            LogVerboseMessage("Before invoking UpdateUserInfo web service method");
            this.StartStopWatch();
            //LogVerboseMessage("Get current user information for:{0}", login); 
            string notes1 = (string.IsNullOrEmpty(notes)) ? "" : notes;
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            client.UpdateUserInfo(login, displayname, email, notes1);
            LogVerboseMessage("After invoking UpdateUserInfo web service method");
            this.StopStopWatch();
        }
        private void StartStopWatch()
        {
            if (_sw == null)
                _sw = new Stopwatch();
            if (_sw.IsRunning) _sw.Stop();
            _sw.Reset();
            _sw.Start();
        }
        private void StopStopWatch()
        {
            if (_sw == null) return;
            if (_sw.IsRunning) _sw.Stop();
            LogVerboseMessage("Time elapsed:{0} ms", _sw.ElapsedMilliseconds);
            _sw.Reset();
        }


        public SPUser GetSiteUser(string sitecollectionurl, string login)
        {
            string url2 = this.GetContainerWebFromDocLibUrl(sitecollectionurl);
            SPWebShell.SPUserGroup.UserGroup client = new SPWebShell.SPUserGroup.UserGroup();
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            client.Url = url2 + "_vti_bin/UserGroup.asmx";
            client.Credentials = System.Net.CredentialCache.DefaultCredentials;
            LogVerboseMessage("Before invoking UpdateUserInfo web service method");
            this.StartStopWatch();
            XmlNode ndUser=client.GetUserInfo(login);
            XmlNode ndUserElement = ndUser.ChildNodes[0];
            SPUser user = XmlToUser(ndUserElement);
            return user;     
        }


        public bool CheckinFile(string urlAbsolute,string comments, int checkintype)
        {
            switch(checkintype)
            {
                case 0:
                case 1:
                case 2:
                    break;
                default:
                    throw new ArgumentException("Checkin type can be 0,1 or 2");
            }
            string url2 = this.GetContainerWebFromDocLibUrl(urlAbsolute);
            SPWebShell.SPLists.Lists lists = new SPWebShell.SPLists.Lists();
            lists.Credentials = System.Net.CredentialCache.DefaultCredentials;
            url2 = url2.Trim();
            if (!url2.EndsWith("/")) url2 = url2 + "/";
            lists.Url = url2 + "_vti_bin/lists.asmx";
            return lists.CheckInFile(urlAbsolute, comments, checkintype.ToString());
        }

        public bool CheckinFile(SPDoclibLocation location, int spid, string comments, int checkintype)
        {
            throw new NotImplementedException();            
        }
    }
}
